
#!/usr/bin/env python3
#coding:utf-8
# 
# 代码来源： Python学习群

__author__ = 'xmxoxo'




import requests
from bs4 import BeautifulSoup
import os
import re
import time
import sys
import getopt
import signal


'''
安装模块

pip install beautifulsoup4
pip install html5lib
pip install lxml
pip install lxml --upgrade

'''
Hostreferer = {
    'User-Agent':'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
    'Referer':'http://www.mzitu.com'
}
Picreferer = {
    'User-Agent':'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
    'Referer':'http://i.meizitu.net'
}

#中断处理 2019/1/10 add by Echo
def signal_handler(signal,frame,isexit = 1):
    print()
    print('You pressed Ctrl+C!\n用户中断，退出。')
    if isexit:
        sys.exit(0)
 


#获得图集最大页数和名称
def get_page_name(url):
    try:
        html = get_html(url)
        soup = BeautifulSoup(html, 'lxml')
        span = soup.findAll('span')
        title = soup.find('h2', class_="main-title")
        return span[10].text, title.text
    except Exception as s:
        pass
        return None

def get_html(url):#获得页面html代码
    try:
        req = requests.get(url, headers=Hostreferer)
        html = req.text
        return html
    except Exception as s:
        pass
        return ''

def get_img_url(url, name):
    try:
        html = get_html(url)
        soup = BeautifulSoup(html, 'lxml')
        img_url = soup.find('img', alt= name)
        return img_url['src']
    except Exception as s:
        pass
        return ''

def save_img(img_url, count, name):
    try:
        print('开始下载图片: %s,%d' %(img_url,count))

        new_name = rename(name)
        fn = new_name+'/'+str(count)+'.jpg'

        if not os.path.exists(fn):
            req = requests.get(img_url, headers=Picreferer)

            with open(fn, 'wb') as f:
                f.write(req.content)

            ##保存图片后停止1秒
            time.sleep(1)
            return 1
        else:
            print('图片已下载，跳过!')
            return 0
    except Exception as s:
        #print(s)
        pass
        return 0


def rename(name):
    #try:
    rstr = r'[\/\\\:\*\?\<\>\|]'
    new_name = re.sub(rstr, "", name)
    return new_name
    #except Exception as s:
    #    pass
        #return name


def save_one_atlas(old_url):
    try:
        page, name = get_page_name(old_url)
        new_name = rename(name)
        if not os.path.exists(new_name):
            os.mkdir(new_name)
        
        print("图集 --[%s]-- 开始保存" % (name) )
        for i in range(1, int(page)+1):
            url = old_url + "/" + str(i)
            img_url = get_img_url(url, name)
            #print(img_url)
            print('正在保存第%s张图片：' % (str(i)))
            ret = save_img(img_url, i, name)
            #2019/1/10 某个图片已保存 可以跳过这个图集
            if not ret:
                break
        print("图集-[%s] 保存完成" % (name) )
    except Exception as s:
        pass


#获取列表清单
def get_atlas_list(url):
    try:
        req = requests.get(url, headers=Hostreferer)
        soup = BeautifulSoup(req.text, 'lxml')
        #print('soup:')
        #print (soup)
        atlas = soup.find_all(attrs={'class':'lazy'})
        atlas_list = []
        #print(atlas)
        for atla in atlas:
            atlas_list.append(atla.parent['href'])
        return atlas_list
    except Exception as s:
        pass

#列表页
def save_one_page(start_url):
    try:
        atlas_url = get_atlas_list(start_url)
        print(atlas_url)
        for url in atlas_url:
            save_one_atlas(url)
    except Exception as s:
        pass


##解析命令行参数 
def getparamter(argv):
    frompage = 0
    topage = 0
    try:
        opts, args = getopt.getopt(argv,"hf:t:",["from=","to="])
        if not opts:
            print ('用法: python meizi.py -f 起始页 -t 结束页')
            print ('usage: python meizi.py -f <from page> -t <end page>')
            sys.exit(2)
        for opt, arg in opts:
            if opt == '-h':
                print ('用法: python meizi.py -f 起始页 -t 结束页')
                print ('usage: python meizi.py -f <from page> -t <end page>')
                sys.exit(0)
            elif opt in ("-f", "--from"):
                if arg.isdigit():
                    frompage = int(arg)
                    topage = frompage + 10
            elif opt in ("-t", "--to"):
                if arg.isdigit():
                   topage = int(arg)

        print ('起始页：', frompage)
        print ('结束页：', topage)
        return frompage, topage
    except getopt.GetoptError:
        print ('用法: python meizi.py -f 起始页 -t 结束页')
        print ('usage: python meizi.py -f <from page> -t <end page>')
        sys.exit(2)


def main(strUrl):
    signal.signal(signal.SIGINT,signal_handler)

    try:
        frompage = 0
        topage = 0
        frompage,topage = getparamter(sys.argv[1:])
        if frompage <= topage and frompage > 0:
            for count in range(frompage, topage ):
                url = strUrl + "page/" + str(count) +"/"
                print(url)
                save_one_page(url)
            print("爬取完成")
    except Exception as s:
        pass

if __name__ == '__main__':
    start_url = "https://www.mzitu.com/"
    main(start_url)
